import scrapy
from flask import current_app
from scrapy import Request
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

from app import evn_
from spiders.items import NewsItem
from utils.util import CommonUtil


class NewsIndustryInfoSpider(scrapy.Spider):
    '''
    行业资讯
    https://finance.eastmoney.com/a/ccjxw.html
    '''
    name = 'news_industry_information'

    def start_requests(self):
        start_urls = ['https://finance.eastmoney.com/a/ccjxw.html']
        start_url = 'https://finance.eastmoney.com/a/ccjxw_{pageNo}.html'
        for index in range(2, 25):
            start_urls.append(start_url.format(pageNo=index))

        for url in start_urls:
            yield Request(url, callback=self.parse_item)

    def __init__(self):
        chrome_options = Options()
        chrome_options.add_argument('--disable-gpu')
        chrome_options.add_argument('lang=zh_CN.UTF-8')
        chrome_options.add_argument('headless')  # 无头浏览器
        chrome_options.add_argument('--no-sandbox')  # 必要！！
        chrome_options.add_argument('--disable-dev-shm-usage')  # 必要！！
        prefs = {
            "profile.managed_default_content_settings.images": 2,  # 禁止加载图片
            'permissions.default.stylesheet': 2,  # 禁止加载css
        }
        chrome_options.add_experimental_option("prefs", prefs)
        self.bro = webdriver.Chrome(executable_path=evn_.CHROMEDRIVER_PATH, chrome_options=chrome_options)

    def parse_item(self, response):
        current_app.logger.info("{}".format(response))
        li_list = response.xpath('//*[@id="newsListContent"]/li')
        if li_list:
            for li in li_list:
                href = li.xpath('div')[-1].xpath('p[1]/a/@href').extract_first()
                title = li.xpath('div')[-1].xpath('p[1]/a/text()').extract_first()
                content = li.xpath('div')[-1].xpath('p[2]/@title').extract_first()
                publish_time = li.xpath('div')[-1].xpath('p[3]/text()').extract_first()
                item = NewsItem()
                item['title'] = CommonUtil.optimizeStr(title)
                item['content'] = CommonUtil.optimizeStr(content)
                item['link'] = href
                if href:
                    item['news_id'] = href.split('/')[-1].split('.')[0]
                else:
                    current_app.logger.error("title:{},content:{},href为空".format(title, content))
                item['publish_time'] = CommonUtil.str2date(publish_time)
                item['source'] = 2
                yield item
